#!/usr/bin/env python

# pybtrss - Downloads bittorrent files from RSS feeds
#
# Copyright (C) 2006 Riku 'Shrike' Lindblad
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

import urllib
import urllib2
import urlparse
import os, os.path
import re
import sys
import logging
import xml.sax

import socket
socket.setdefaulttimeout(10) # time out in 10 seconds

try:
    import feedparser
except ImportError:
    print "Please install Feedparser from http://www.feedparser.org/ or from your distro repository"
    sys.exit(1)
try:
    import yaml
except ImportError:
    print "Please install PyYAML from http://pyyaml.org/wiki/PyYAML or from your distro repository"
    sys.exit(1)

class pybtrss(object):
    """Bittorrent RSS feed scanner & downloader"""

    # as per http://freedesktop.org/wiki/Standards_2fconfig_2dspec
    _CONFIGDIR = os.path.expanduser("~/.config/pybtrss")

    _config = None
    _cache = None

    def __init__(self):
        try:
            logging.basicConfig(level=logging.INFO,
                                format='%(asctime)s %(levelname)-8s %(message)s',
                                filename=os.path.join(sys.path[0], 'pybtrss.log'),
                                filemode="a",
                                datefmt='%Y-%m-%d %H:%M:%S')
        except TypeError:
            # For pre-2.4 pythons5C
            logger = logging.getLogger() # root logger
            hdlr = logging.FileHandler(os.path.join(sys.path[0], 'pybtrss.log'))
            formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
            hdlr.setFormatter(formatter)
            logger.addHandler(hdlr)
            logger.setLevel(logging.INFO)
            
        # create config directory in .config if needed
        if not os.path.exists(self._CONFIGDIR):
            os.makedirs(self._CONFIGDIR)

        self.load_config()

        # define hook types
        self.hooks = {            
            'fetched torrent': [], # (feed, [releases])
            }

    def add_hook(self, hooktype, function):
        """Add a new hook function of the given type"""
        
        if hooktype in self.hooks.keys():
            self.hooks[hooktype].append(function)
        else:
            logging.error("Invalid hook type '%s'" % hooktype)        

    def _run_hooks(self, name, *args):
        """Hook runner helper function"""
        
        logging.debug("Running hook %s" % name)
        
        for hook in self.hooks[name]:
            hook(*args)
        
    def run(self):
        self.check_feeds()
        self.save_config()
        logging.debug("Checking done")

    def is_dir_valid(self, directory):
        # check that the destination directory exists
        return os.path.exists(os.path.expanduser(self._config['torrent_dir']))

    def check_feeds(self):
        """Check feeds if there is anything new to get"""

        # check that the destination directory exists
        #if not os.path.exists(os.path.expanduser(self._config['torrent_dir'])):
        #    raise IOError("Torrent directory %s not found" % os.path.expanduser(self._config['torrent_dir']))
        if not self.is_dir_valid(self._config['torrent_dir']):
            raise IOError("Torrent directory %s not found" % os.path.expanduser(self._config['torrent_dir']))
        
        # go through the feeds
        for feed in self._config['feeds']:
            feedcfg = self._config['feeds'][feed]

            # new feeds need their cache initialized
            if feedcfg.has_key('torrent_dir'):
                if not self.is_dir_valid(feedcfg['torrent_dir']):
                    raise IOError("Feed specific torrent directory %s not found for feed %s" % (os.path.expanduser(feedcfg['torrent_dir']), feed))
                
            if not self._cache.has_key(feed):
                logging.debug("Initialized cache for feed %s" % feed)
                self._cache[feed] = {'seen':[], 'etag':None, 'modified':None}
            
            url = feedcfg['url']

            # use basic auth when needed
            if feedcfg.has_key('username') and feedcfg.has_key('password'):
                #auth.add_password(feedcfg['realm'], feedcfg['host'], feedcfg['username'], feedcfg['password'])
                url = self.passwordize(url, feedcfg['username'], feedcfg['password'])

            logging.debug("Checking feed %s (%s)", feed, url)
            
            # check etags and last modified -headers
            etag = self._cache[feed].get('etag', None)
            if etag:
                logging.debug("Sending etag %s for feed %s" % (etag, feed))
            modified = self._cache[feed].get('modified', None)
            if modified:
                logging.debug("Sending last-modified %s for feed %s" % (etag, feed))

            # get the feed & parse
            try:
                rss = feedparser.parse(url, etag=etag, modified=modified)
            except IOError:
                logging.error("IOError when loading feed %s", feed)
                continue # and try again later

            try:
                # status checks
                if rss.status == 304:
                    logging.debug("Feed %s hasn't changed, skipping" % feed)
                    continue
                elif rss.status == 401:
                    logging.warning("Authentication needed for feed %s: %s", feed, rss.headers['www-authenticate'])
                    continue
                elif rss.status == 404:
                    logging.error("Feed %s not found", feed)
                    continue
                elif rss.status == 500:
                    logging.error("Internal server exception on feed %s", feed)
                    continue
            except urllib2.URLError:
                logging.error("URLError on feed %s", feed)
                continue
            except AttributeError, e:
                ex = rss['bozo_exception']
                if ex == feedparser.NonXMLContentType:
                    print "feedparser.NonXMLContentType"
                    continue
                elif ex == xml.sax._exceptions.SAXParseException:
                    print "xml.sax._exceptions.SAXParseException"
                    continue
                elif ex == urllib2.URLError:
                    print "urllib2.URLError"
                    continue
                else:
                    logging.error("ERROR MATCHING ERRORISSIMO")

            if rss['bozo']:
                logging.error(rss)
                logging.error("Bozo feed exception on %s. Is the URL correct?" % feed)
                continue
                      
            # update etag, use last modified if no etag exists
            if rss.etag:
                self._cache[feed]['etag'] = rss.etag.replace("'", "").replace('"', "")
                logging.debug("etag "+self._cache[feed]['etag']+" saved for feed %s", feed)
            elif rss.headers.has_key('last-modified'):
                feedcfg['modified'] = rss.modified
                logging.debug("last modified saved for feed %s", feed)

            self.download_torrents(feed, rss)
                
    def download_torrents(self, feed, rss):
        """Download matching new torrents"""

        feedcfg = self._config['feeds'][feed]

        # pick a download directory
        if feedcfg.has_key("torrent_dir"):
            download_directory = feedcfg["torrent_dir"]
        else:
            download_directory = self._config['torrent_dir'];

        fetched_torrents = []

        # get patterns for release matching
        (patterns, ignores) = self.get_patterns(feed)

        # check the feed for new items
        for entry in rss.entries:

            self.save_config()

            # fix for crap feeds with no ID
            if not entry.has_key('id'):
                entry['id'] = entry.link
            
            # skip seen entries
            if entry.id in self._cache[feed]["seen"]:
                #logging.debug("Skipped already seen torrent with id %s", entry.id)
                continue

            ignorematch = False
            # ignores
            for pattern in ignores:
                if re.search(pattern, entry.title, re.IGNORECASE|re.UNICODE):
                    logging.info("%s matched ignore pattern %s on %s" % (entry.title, pattern, feed))
                    # add to seen list
                    self._cache[feed]["seen"].append(str(entry.id))
                    ignorematch = True
                    break

            if ignorematch: continue

            # match title to given patterns            
            for pattern in patterns:
                # if pattern matches, download it
                if re.search(pattern, entry.title, re.IGNORECASE|re.UNICODE):                   
                    logging.info("%s matched pattern %s on %s" % (entry.title, pattern, feed))

                    # Fixes for "interesting" feed structures
                    if entry.link.startswith("http://www.mininova.org/tor/"):
                        entry.link = entry.link.replace('tor', 'get')
                    elif entry.link.startswith("http://www.torrentspy.com/torrent/"):
                        m = re.match("http://www.torrentspy.com/torrent/([\d]+)/", entry.link)
                        torrent_id = m.group(1)
                        entry.link = "http://www.torrentspy.com/download.asp?id=%s" % torrent_id

                    # Use basic auth when needed
                    if feedcfg.has_key('username') and feedcfg.has_key('password'):
                        logging.debug("Using basic auth for retrieval")
                        entry.link = self.passwordize(entry.link, feedcfg['username'], feedcfg['password'])
                    
                    # get torrent
                    # urllib2 is too smart here, it borks on basic auth urls
                    logging.debug("Loading torrent...")
                    try:
                        f = urllib.urlopen(entry.link)
                        torrent = f.read()
                        f.close()
                    except IOError:
                        logging.error("IOError when loading torrent %s", entry.link)
                        continue # try again later.                        
                        
                    logging.debug("Done")

                    # valid torrent files start with an announce block
                    if not torrent.startswith("d8:announce"):
                        logging.error("Invalid torrent: %s", entry.link)
                        continue

                    # create a proper filename
                    # TODO: Add feed name here too?
                    filename = entry.title.replace(" ", "_") + ".torrent"
                    filename = filename.replace("/", "_")
                    filename = filename.encode('iso8859-1', 'ignore') # Damn \u200b -character, how I loathe thee
                    # save it
                    destfile = os.path.join(os.path.expanduser(download_directory), filename)

                    try:
                        if not os.path.exists(destfile):
                            f = file(destfile, 'w')
                            f.write(torrent)
                            f.close()
                        else:
                            logging.error("File '%s' already exists", destfile)
                            self._cache[feed]["seen"].append(str(entry.id)) # just in case?
                            continue # Don't run hooks on existing torrents
                    except IOError:
                        logging.error("IOError when saving torrent %s" % destfile)
                        continue # And don't run hooks on broken torrents

                    fetched_torrents.append(entry.title)
                    # add to seen list
                    self._cache[feed]["seen"].append(str(entry.id))
                    
        # run hooks for torrents fetched from this feed
        if fetched_torrents:
            try:
                self._run_hooks('fetched torrent', feed, fetched_torrents)
            except:
                logging.error("Error when running hooks")
                raise

    def get_patterns(self, feed):
        """Get all patterns specific for the given feed + all global patterns"""

        patterns = self._config['feeds'][feed].get('patterns', [])
        patterns += self._config.get('global_patterns', [])

        ignore =  self._config['feeds'][feed].get('ignores', [])        
        ignore += self._config.get('global_ignores', [])
        
        return patterns, ignore

    def load_config(self):
        """Load configuration"""

        # Cache data
        cachefile = os.path.join(self._CONFIGDIR, 'cache')
        if os.path.exists(cachefile):
            try:
                self._cache = yaml.safe_load(file(cachefile))
            except yaml.scanner.ScannerError:
                logging.error("Cache file corrupted! Creating a new one.")
                self._cache = {}
            except yaml.parser.ParserError:
                logging.error("Cache file corrupted! Creating a new one.")
                self._cache = {}
        else:
            self._cache = {}

        # config data
        configfile = os.path.join(sys.path[0], 'config')
        if os.path.exists(configfile):
            self._config = yaml.safe_load(file(configfile))
        else:
            self.create_example_config()
            logging.warning("No configuration file found, I created an example config for you")
            sys.exit(0)

    def save_config(self):
        """Save configuration"""
        
        f = file(os.path.join(self._CONFIGDIR, 'cache'), 'w')
        yaml.dump(self._cache, f)
        f.close()    

    def passwordize(self, url, user, password):
        """Add username and password to url"""

        parts = list(urlparse.urlsplit(url))
        parts[1] = user+":"+password+"@"+parts[1]
        url = urlparse.urlunsplit(parts)
        return url        

    def create_example_config(self):
        """Create an example config file for the user"""
        
        conf= yaml.load("""
        feeds:
          sample:
            url: http://private.example.net/rss/
            username: example
            password: example
            patterns:
            - ^battlestar banana
            - ^doctor what
            - ^found
            - ^42
          another sample:
            url: http://example.net/rss/
            patterns:
            - ^starport sp1
            - ^the 5500
            ignores:
            - ^cabin
          yet another sample:
            url: http://example.net/rss/
        torrent_dir: ~/
        global_ignores:
          - polish.polka.group
        global_patterns:
          - ^cabin
        """)

        f = file(os.path.join(sys.path[0], 'config'), 'w')
        yaml.dump(conf, f)
        f.close()

def mail_fetched(feed, releases):
    """Mail info about new torrents"""

    email = "user@host.tld"

    subject = "New releases from %s" % feed
    mailcmd = "mail -s '%s' %s" % (subject, email)
    f = os.popen(mailcmd, 'w')
    for release in releases:
        f.write("%s\n" % release.encode('iso8859-1', 'ignore'))
    f.close()

if __name__ == "__main__":
    bt = pybtrss()
    # set the correct email address in the mail_fetched function and uncomment
    #bt.add_hook('fetched torrent', mail_fetched)
    bt.run()

